{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'html_out/_nfu_学校要闻.html'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mFileNotFoundError\u001B[0m                         Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-8-d6dfa017a174>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      8\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      9\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 10\u001B[1;33m \u001B[1;32mwith\u001B[0m \u001B[0mopen\u001B[0m \u001B[1;33m(\u001B[0m\u001B[1;34m\"html_out/_nfu_学校要闻.html\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mencoding\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;34m\"utf8\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mmode\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;34m\"w\"\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mfp\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     11\u001B[0m     \u001B[0mfp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mwrite\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mr\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mhtml\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mhtml\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     12\u001B[0m \u001B[1;32mwith\u001B[0m \u001B[0mopen\u001B[0m \u001B[1;33m(\u001B[0m\u001B[1;34m\"html_out/_nfu_学校要闻.html\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mencoding\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;34m\"utf8\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mmode\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;34m\"r\"\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mfp\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mFileNotFoundError\u001B[0m: [Errno 2] No such file or directory: 'html_out/_nfu_学校要闻.html'"
     ]
    }
   ],
   "source": [
    "from requests_html import HTMLSession\n",
    "import requests_html\n",
    "import pandas as pd\n",
    "import urllib.parse\n",
    "\n",
    "session = HTMLSession()\n",
    "r = session.get(\"https://www.nfu.edu.cn/xxyw/index.htm\")\n",
    "\n",
    "\n",
    "with open (\"html_out/_nfu_学校要闻.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)\n",
    "with open (\"html_out/_nfu_学校要闻.html\", encoding = \"utf8\", mode = \"r\") as fp:\n",
    "    html_load = fp.read()\n",
    "\n",
    "parsed = requests_html.soup_parse(html_load)\n",
    "parsed\n",
    "\n",
    "base_url = r.url\n",
    "nfu_urlparse = urllib.parse.urlparse(base_url)\n",
    "nfu_urlparse\n",
    "\n",
    "\n",
    "list_URL  = [urllib.parse.urlunparse\\\n",
    "([nfu_urlparse.scheme,nfu_urlparse.netloc,'/'+ nfu_urlparse.path.split('/')[1] +'/' + detail_url,'','',''])\\\n",
    "for detail_url in parsed.xpath('//div[@class=\"news_title\"]/a/@href')]\n",
    "list_URL\n",
    "\n",
    "df = pd.DataFrame( {\n",
    "         \"标题\": parsed.xpath('//div[@class=\"news_title\"]/a/@title'),\n",
    "         \"链结\": list_URL,\n",
    "         \"日期\": parsed.xpath('//font[@class=\"right-more\"]/text()'),\n",
    "     } )\n",
    "df\n",
    "\n",
    "for i in range(1,100):\n",
    "    r = session.get('https://www.nfu.edu.cn/xxyw/index'+str(i)+'.htm')\n",
    "    if r.status_code != 200:\n",
    "        print(i)\n",
    "        break\n",
    "\n",
    "url_group = ['https://www.nfu.edu.cn/xxyw/index'+str(i)+'.htm' for i in range(1,100)]\n",
    "url_group\n",
    "\n",
    "url_group.insert(0,'https://www.nfu.edu.cn/xxyw/index.htm')\n",
    "url_group\n",
    "urllib.parse.urlparse(url_group[0]).path\n",
    "\n",
    "\n",
    "for url in url_group:\n",
    "    r = session.get(url)\n",
    "#     print(r.html.html)\n",
    "    path = urllib.parse.urlparse(url).path\n",
    "    with open ('html_out/'+path, encoding = \"utf8\", mode = \"w\") as fp:\n",
    "        fp.write(r.html.html)\n",
    "\n",
    "dict_xpath = {\n",
    "    '链接_xpath':'//div[@class=\"news_title\"]/a/@href',\n",
    "    '标题_xpath':'//div[@class=\"news_title\"]/a/@title',\n",
    "    '日期_xpath':'//font[@class=\"right-more\"]/text()'\n",
    "}\n",
    "\n",
    "\n",
    "def pages_content_url(parsed):\n",
    "    list_URL  = [urllib.parse.urlunparse\\\n",
    "                 ([nfu_urlparse.scheme,nfu_urlparse.netloc,'/'+ nfu_urlparse.path.split('/')[1] +'/' + detail_url,'','',''])\\\n",
    "                 for detail_url in parsed.xpath(dict_xpath['链接_xpath'])]\n",
    "    return list_URL\n",
    "\n",
    "\n",
    "import os\n",
    "\n",
    "list_df = []\n",
    "\n",
    "\n",
    "files= os.listdir('html_out/xxyw/')\n",
    "print(files)\n",
    "\n",
    "for html in files:\n",
    "    with open('html_out/xxyw/'+html,encoding='utf8',mode='r') as fp:\n",
    "        html_load = fp.read()\n",
    "        parsed = requests_html.soup_parse(html_load)\n",
    "        list_URL = pages_content_url(parsed)\n",
    "\n",
    "        df1= pd.DataFrame( {\n",
    "         \"标题\": parsed.xpath(dict_xpath['标题_xpath']),\n",
    "         \"链结\": list_URL,\n",
    "         \"日期\": parsed.xpath(dict_xpath['日期_xpath']),\n",
    "        } )\n",
    "        list_df.append(df1)\n",
    "\n",
    "\n",
    "\n",
    "df_all1 = pd.concat(list_df).reset_index().sort_values(by='日期',ascending=False)\n",
    "display(df_all1)\n",
    "\n",
    "with pd.ExcelWriter('data_out/nfu_学校要闻.xlsx',mode='w',engine=\"openpyxl\") as writer:\n",
    "            df_all1.to_excel(writer, sheet_name='学校要闻')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}